import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import pickle # import cpickle as pickle
def data_save(data,filename):
f = open(filename, "wb") # *.dat
pickle.dump(data, f)
f.close()
def data_load(filename):
return pickle.load(open(filename, "rb"))
dpi = 100
def plot_lr(x, value_history,step=600, name='plot_lr'):
id_saved = int(step/10) # each 10 step
max_step = value_history[0,-1,0]
print('step=',step,'max_step=',max_step)
plt.figure(figsize=[12,5])
plt.subplot(121)
plt.semilogx(x,value_history[:,id_saved,1],'b-')
plt.plot(x,value_history[:,id_saved,3],'r-')
#plt.xlim([0,10])
plt.xlabel('learning rate')
plt.ylabel('loss at step=%d'% step);
plt.legend(('train','test'))
plt.subplot(122)
plt.semilogx(x,value_history[:,id_saved,2],'b-')
plt.plot(x,value_history[:,id_saved,4],'r-')
#plt.xlim([0,10])
plt.xlabel('learning rate')
plt.ylabel('accuracy at step=%d'%step);
plt.legend(('train','test'),loc=0)
# always call tight_layout before saving ;)
plt.tight_layout()
plt.savefig("./%s.png"%name, dpi=dpi)
plt.savefig("./%s.pdf"%name)
def plot_lr2(x, vhs,step=600, name='plot_lr2'):
vh = np.mean(vhs, axis=0)
vh_min = np.min(vhs, axis=0)
vh_max = np.max(vhs, axis=0)
id_saved = int(step/10) # each 10 step
max_step = vh[0,-1,0]
print('step=',step,'max_step=',max_step)
plt.figure(figsize=[12,5])
plt.subplot(121)
plt.semilogx(x,vh[:,id_saved,1],'b.-')
plt.fill_between(x, vh_min[:,id_saved,1],vh_max[:,id_saved,1],color='b',alpha=0.1)
plt.plot(x,vh[:,id_saved,3],'r.-')
plt.fill_between(x, vh_min[:,id_saved,3],vh_max[:,id_saved,3],color='r',alpha=0.1)
#plt.xlim([0,10])
plt.xlabel('learning rate')
plt.ylabel('loss at step=%d'% step);
plt.legend(('train','test'))
plt.subplot(122)
plt.semilogx(x,vh[:,id_saved,2],'b.-')
plt.fill_between(x, vh_min[:,id_saved,2],vh_max[:,id_saved,2],color='b',alpha=0.1)
plt.plot(x,vh[:,id_saved,4],'r.-')
plt.fill_between(x, vh_min[:,id_saved,4],vh_max[:,id_saved,4],color='r',alpha=0.1)
#plt.xlim([0,10])
plt.xlabel('learning rate')
plt.ylabel('accuracy at step=%d'%step);
plt.legend(('train','test'))
# always call tight_layout before saving ;)
plt.tight_layout()
plt.savefig("./%s.png"%name, dpi=dpi)
plt.savefig("./%s.pdf"%name)
def plot_bnsplr(x, lr_list, vhs,step=600, name='plot_bnsplr'):
id_saved = int(step/10) # saved each 10 step
fig,ax = plt.subplots(2, 2, figsize=(16, 8))
for i in range(len(values)):
vh = vhs[i]
#print(vh[:,id_saved,0])
max_step = vh[0,-1,0]
ax[0,0].semilogx(x,vh[:,id_saved,1])
ax[0,1].semilogx(x,vh[:,id_saved,2])
ax[1,0].semilogx(x,vh[:,id_saved,3])
ax[1,1].semilogx(x,vh[:,id_saved,4])
print('step=',step,'max_step=',max_step)
ax[0,0].set_ylabel('Train')
ax[1,0].set_ylabel('Test')
ax[0,0].set_title('Loss at step=%d' % step)
ax[0,1].set_title('Accuracy at step=%d' % step)
ax[1,0].set_xlabel('learning rate')
ax[1,1].set_xlabel('learning rate')
plt.legend(['eps_a=%g' % eps_a for eps_a in lr_list])
# always call tight_layout before saving ;)
plt.tight_layout()
plt.savefig("./%s.png"%name, dpi=dpi)
plt.savefig("./%s.pdf"%name)
def plot_bnsplr2(x, lr_list, values,step=600, name='plot_bnsplr2'):
id_saved = int(step/10) # saved each 10 step
fig,ax = plt.subplots(2, 2, figsize=(16, 8))
for i in range(len(values)):
vhs = values[i]
vh = np.mean(vhs,axis=0)
vh_min = np.min(vhs,axis=0)
vh_max = np.max(vhs,axis=0)
#print(vh[:,id_saved,0])
max_step = vh[0,-1,0]
ax[0,0].semilogx(x,vh[:,id_saved,1])
ax[0,0].fill_between(x,vh_min[:,id_saved,1],vh_max[:,id_saved,1],alpha=0.1)
ax[0,1].semilogx(x,vh[:,id_saved,2])
ax[0,1].fill_between(x,vh_min[:,id_saved,2],vh_max[:,id_saved,2],alpha=0.1)
ax[1,0].semilogx(x,vh[:,id_saved,3])
ax[1,0].fill_between(x,vh_min[:,id_saved,3],vh_max[:,id_saved,3],alpha=0.1)
ax[1,1].semilogx(x,vh[:,id_saved,4])
ax[1,1].fill_between(x,vh_min[:,id_saved,4],vh_max[:,id_saved,4],alpha=0.1)
print('step=',step,'max_step=',max_step)
ax[0,0].set_ylabel('Train')
ax[1,0].set_ylabel('Test')
ax[0,0].set_title('Loss at step=%d' % step)
ax[0,1].set_title('Accuracy at step=%d' % step)
ax[1,0].set_xlabel('learning rate')
ax[1,1].set_xlabel('learning rate')
plt.legend(['eps_a=%g' % eps_a for eps_a in lr_list])
# always call tight_layout before saving ;)
plt.tight_layout()
plt.savefig("./%s.png"%name, dpi=dpi)
plt.savefig("./%s.pdf"%name)
%%writefile mnist_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_gd()
taskname = 'mnist_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,1,20), max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m1/mnist_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
plot_lr(x, np.mean(value_history, axis=0),step=590, name='./figures/mnist_m1_gdlr')
plot_lr2(x, value_history,step=590, name='./figures/mnist_m1_gdlr2')
%%writefile mnist_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_bn()
taskname = 'mnist_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnlr_T%d.dat' % i)
x = s[0]
value_history.append(s[1])
plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m1_bnlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m1_bnlr2')
%%writefile mnist_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = 0.001 # 1.5, 1.0, 0.5, 0.1, 0.01, 0.001
taskname = 'mnist_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
vh = []
for i in range(5): # only one test
s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
vh.append(s[1])
#values.append(np.mean(vh, axis=0))
values.append(vh)
step = 600 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=600,name='./figures/mnist_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=600,name='./figures/mnist_m1_bnsplr2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m1/mnist_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
x_bnsp = x
vhs_bnsp = values
step = 600 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 59
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.7])
plt.xlabel('learning rate')
plt.ylabel('loss')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.94,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.48), loc=2, borderaxespad=0.)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m1_compare.png", dpi=150)
plt.savefig("./figures/mnist_m1_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.7])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.94,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.48), loc=2, borderaxespad=0.)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m1_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m1_compare_test.pdf")
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
for lrab in [1.0, 0.1, 0.01, 0.001]:
for aug in [0.01,0.1,1,10]:
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = lrab
model.set_scaling(weight_aug=aug)
taskname = 'mnist_m1_sca_bnsplr_ab%g_aug%g_T%d' % (lrab,aug,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-4,4,40), max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)
values = []
ilr = 0
plt.figure(figsize=[12,8])
for lrab in [1.0, 0.1, 0.01, 0.001]:
ilr += 1
#for aug in [0.1,1,10]: #[0.01,0.1,1,10]:
for aug in [0.01,0.1,1,10]:
vh = []
for i in range(5):
taskname = 'mnist_m1_sca_bnsplr_ab%g_aug%g_T%d' % (lrab,aug,i)
datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
s = data_load(datafile)
x = s[0]
x = s[0] / aug**2
vh.append(s[1])
plt.subplot(2,2,ilr)
plt.semilogx(x,np.mean(vh,axis=0)[:,10,1])
plt.xlabel('learning rate / aug^2')
plt.ylabel('loss')
plt.title('lrab=%g'%lrab)
#plt.legend(['aug=%g'%aug for aug in [0.1,1,10]])
plt.legend(['aug=%g'%aug for aug in [0.01,0.1,1,10]])
plt.tight_layout()
name = './figures/mnist_m1_sca_bnsplr'
plt.savefig("%s.png"%name, dpi=dpi)
plt.savefig("%s.pdf"%name)
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
for lrab in [1.0]: # , 0.1, 0.01, 0.001
for aug in np.logspace(-2,2,17):# [0.01,0.1,1,10]:
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = lrab
model.set_scaling(weight_aug=aug)
taskname = 'mnist_m1_sca_bnsplr_add_ab%g_aug%g_T%d' % (lrab,aug,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-4,4,40)*aug**2, max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)
# killed -- so slowly
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
for lrab in [1.0]: # , 0.1, 0.01, 0.001
for aug in np.logspace(-2,2,9):# [0.01,0.1,1,10]:
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = lrab
model.set_scaling(weight_aug=aug)
taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,3,20)*aug**2, max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
for lrab in [1.0]: # , 0.1, 0.01, 0.001
for aug in np.logspace(-2,2,9):# [0.01,0.1,1,10]:
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = lrab
model.set_scaling(weight_aug=aug)
taskname = 'mnist_m1_sca_bnsplr_add3_ab%g_aug%g_T%d' % (lrab,aug,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-4,4,20)*aug**2, max_step=600,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)
values = []
ilr = 0
plt.figure(figsize=[10,4])
lrab = 1
aug_list = np.logspace(-2,2,9)
#aug_list = aug_list[0:9:2]
for aug in aug_list:
#for aug in np.logspace(-2,2,17):
vh = []
for i in range(4):
taskname = 'mnist_m1_sca_bnsplr_add3_ab%g_aug%g_T%d' % (lrab,aug,i)
datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
s = data_load(datafile)
x = s[0] / aug**2
vh.append(s[1])
plt.subplot(121)
plt.loglog(x,np.mean(vh,axis=0)[:,60,1])
plt.xlabel('learning rate / aug^2')
#plt.ylim([3e-3,1])
#plt.title('lrab=%g'%lrab)
plt.ylabel('loss')
plt.subplot(122)
plt.semilogx(x,np.mean(vh,axis=0)[:,60,2])
plt.xlabel('learning rate / aug^2')
plt.ylim([0.9,1])
#plt.title('lrab=%g'%lrab)
plt.ylabel('accuracy')
plt.subplot(121)
plt.legend(['aug=%g'%aug for aug in aug_list],bbox_to_anchor=(.5, 0.97), loc=2, borderaxespad=0.)
plt.subplot(122)
plt.legend(['aug=%g'%aug for aug in aug_list],bbox_to_anchor=(.5, 0.8), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/mnist_m1_bnsp_lra1_scaling_V2.png", dpi=100)
plt.savefig("./figures/mnist_m1_bnsp_lra1_scaling_V2.pdf")
values = []
ilr = 0
plt.figure(figsize=[6,4])
lrab = 1
stepid = 60
vh = []
for i in range(5):
aug = 1
taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
s = data_load(datafile)
x = s[0] / aug**2
vh.append(s[1])
vh0 = np.mean(vh,axis=0)[:,stepid,1]
for aug in np.logspace(-2,2,9):
vh = []
for i in range(5):
taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
s = data_load(datafile)
x = s[0] / aug**2
vh.append(s[1])
y = np.mean(vh,axis=0)[:,stepid,1]
#plt.loglog(x,y/vh0)
plt.loglog(x,y)
plt.xlabel('learning rate / aug^2')
plt.title('lrab=%g'%lrab)
plt.legend(['aug=%g'%aug for aug in np.logspace(-2,2,9)])
%%writefile mnist_m2_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_gd()
taskname = 'mnist_m2_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,1,20), max_step=6000, # 600
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
print('Over')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m2m2/mnist_m2_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m2_gdlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m2_gdlr2')
%%writefile mnist_m2_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_bn()
taskname = 'mnist_m2_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=6000, # 600
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
value_history = []
for i in range(7): # add 2 test
s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m2_bnlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m2_bnlr2')
%%writefile mnist_m2_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_bn_split()
model.learning_rate_ab = 0.001 # 1.0, 0.1, 0.01, 0.001
taskname = 'mnist_m2_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=600, # 6000
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 600 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=600,name='./figures/mnist_m2_bnsplr')
plot_bnsplr2(x, lr_list, values,step=600,name='./figures/mnist_m2_bnsplr2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m2m2/mnist_m2_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
x_bnsp = x
vhs_bnsp = values
step = 600 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 60
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.4])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
plt.grid()
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m2_compare.png", dpi=150)
plt.savefig("./figures/mnist_m2_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.4])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m2_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m2_compare_test.pdf")
%%writefile ./script/mnist_m4_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model4_mnist_gd()
taskname = 'mnist_m4_gdlr_add_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,5,40), max_step=6000,
#lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)
print('Over')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_gdlr_add_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m4_gdlr_add')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m4_gdlr_add2')
%%writefile ./script/mnist_m4_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model4_mnist_bn()
taskname = 'mnist_m4_bnlr_add_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,5,40), max_step=6000,
#lr_list=np.logspace(-3,3,40), max_step=6000, # 'mnist_m4_bnlr_T%d' % i
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnlr_add_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m4_bnlr_add')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m4_bnlr_add2')
%%writefile ./script/mnist_m4_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
for i in range(5):
for lrab in [1.0, 0.1, 0.01, 0.001]:
tf.reset_default_graph()
model = Model4_mnist_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'mnist_m4_bnsplr_add_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,7,40), max_step=6000,
#lr_list=np.logspace(-3,3,40), max_step=6000, # 'mnist_m4_bnsplr_ab%g_T%d'
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnsplr_add_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 6000 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/mnist_m4_bnsplr_add')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/mnist_m4_bnsplr_add2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_gdlr_add_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnlr_add_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnsplr_add_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
x_bnsp = x
vhs_bnsp = values
step = 600 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 60
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
plt.grid()
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m4_compare.png", dpi=150)
plt.savefig("./figures/mnist_m4_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/mnist_m4_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m4_compare_test.pdf")
%%writefile fmnist_m1_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_gd()
taskname = 'fmnist_m1_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,1,20), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 600
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m1_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m1_gdlr2')
%%writefile fmnist_m1_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_bn()
taskname = 'fmnist_m1_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)
value_history = []
for i in range(4):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnlr_T%d.dat' % i)
x = s[0]
value_history.append(s[1])
step = 600
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m1_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m1_bnlr2')
%%writefile fmnist_m1_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model_mnist_bn_split()
model.learning_rate_ab = 0.001 # 1.0, 0.1, 0.01, 0.001
taskname = 'fmnist_m1_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
step = 1200 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m1_bnsplr2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(4):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnlr_T%d.dat' % i)
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
vhs_bnsp = values
step = 1200 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.0])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m1_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m1_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.83,0.94])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m1_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m1_compare_test.pdf")
%%writefile fmnist_m2_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_gd()
taskname = 'fmnist_m2_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
test= Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,1,20), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1200 # 1200
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m2_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m2_gdlr2')
%%writefile fmnist_m2_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_bn()
taskname = 'fmnist_m2_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1200 # 1200
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m2_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m2_bnlr2')
%%writefile fmnist_m2_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for lrab in [1.0, 0.1, 0.01, 0.001]:
for i in range(5):
tf.reset_default_graph()
model = Model2_mnist_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'fmnist_m2_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=1200,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 1200 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m2_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m2_bnsplr2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(4):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnlr_T%d.dat' % i)
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
vhs_bnsp = values
step = 1200 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.2])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,0.925])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m2_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m2_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.4,1.2])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,0.86])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m2_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m2_compare_test.pdf")
%%writefile fmnist_m3_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model3_mnist_gd()
taskname = 'fmnist_m3_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,1,20), max_step=6000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 600 # 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m3_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m3_gdlr2')
%%writefile fmnist_m3_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
tf.reset_default_graph()
model = Model3_mnist_bn()
taskname = 'fmnist_m3_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=12000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 12000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m3_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m3_bnlr2')
%%writefile fmnist_m3_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for lrab in [1.0, 0.1, 0.01, 0.001]:
for i in range(5):
tf.reset_default_graph()
model = Model3_mnist_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'fmnist_m3_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=12000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)
%%writefile ./script/fmnist_m3_bnsplr2.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)
for i in range(5):
for lrab in [1.0, 0.1, 0.01, 0.001]:
tf.reset_default_graph()
model = Model3_mnist_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'fmnist_m3_bnsplr2_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=mnist,
lr_list=np.logspace(-3,3,40), max_step=1200, # re do
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
#lr_list = [1]
for lr in lr_list:
value_history = []
for i in range(2):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnsplr2_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 600 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m3_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m3_bnsplr2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(4):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnlr_T%d.dat' % i)
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnsplr2_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
vhs_bnsp = values
step = 1200 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.,0.8])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,1])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m3_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m3_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,0.8])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,0.93])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.45), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/fmnist_m3_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m3_compare_test.pdf")
%%writefile ./script/cifar10_m1_gdlr.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)
for i in range(5):
tf.reset_default_graph()
model = Model_cifar10_gd()
taskname = 'cifar10_m1_gdlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,1,20), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_gdlr2')
%%writefile ./script/cifar10_m1_gdlr2.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=True)
for i in range(5):
tf.reset_default_graph()
model = Model_cifar10_gd()
taskname = 'cifar10_m1_gdlr2_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,1,20), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr2_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_gdlr_norm')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_gdlr_norm2')
%%writefile ./script/cifar10_m1_bnlr.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)
for i in range(5):
tf.reset_default_graph()
model = Model_cifar10_bn()
taskname = 'cifar10_m1_bnlr_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,3,40), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_bnlr2')
%%writefile ./script/cifar10_m1_bnlr2.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=True)
for i in range(5):
tf.reset_default_graph()
model = Model_cifar10_bn()
taskname = 'cifar10_m1_bnlr2_T%d' % i
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,3,40), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr2_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_bnlr_norm')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_bnlr_norm2')
%%writefile ./script/cifar10_m1_bnsplr.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)
for i in range(5):
for lrab in [1.0, 0.1, 0.01, 0.001]:
tf.reset_default_graph()
model = Model_cifar10_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'cifar10_m1_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,3,40), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 1000 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/cifar10_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/cifar10_m1_bnsplr2')
%%writefile ./script/cifar10_m1_bnsplr2.py
from ModelAndTest import *
from Cifar10 import *
dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)
for i in range(5):
for lrab in [1.0, 0.1, 0.01, 0.001]:
tf.reset_default_graph()
model = Model_cifar10_bn_split()
model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
taskname = 'cifar10_m1_bnsplr2_ab%g_T%d' % (model.learning_rate_ab, i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,3,40), max_step=1000,
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr2_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 1000 # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/cifar10_m1_bnsplr_norm')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/cifar10_m1_bnsplr_norm2')
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr_T%d.dat' % (i+0))
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
vhs_bnsp = values
step = 1000 # 1-epoch 600 step
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
save_id = 100
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()
col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
plt.savefig("./figures/cifar10_m1_compare.png", dpi=150)
plt.savefig("./figures/cifar10_m1_compare.pdf")
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()
col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.grid()
plt.tight_layout()
plt.savefig("./figures/cifar10_m1_compare_test.png", dpi=150)
plt.savefig("./figures/cifar10_m1_compare_test.pdf")
%%writefile ./script/mnist_m5_gdlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
dataset = mnist
for i in range(5):
for hidden_size in [10,100,1000]:
tf.reset_default_graph()
model = Model5_mnist_gd(hidden_size=hidden_size)
taskname = 'mnist_m5_h%d_gdlr_T%d' % (hidden_size,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,6,40), max_step=6000,
#lr_list=np.logspace(-3,5,40), max_step=6000,
#lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')
for hidden_size in [10,100,1000]:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_gdlr_T%d.dat' % (hidden_size,i+0))
x = s[0]
value_history.append(s[1])
step = 6000
print(hidden_size)
#plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m5_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m5_gdlr')
%%writefile ./script/mnist_m5_bnlr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
dataset = mnist
for i in range(5):
for hidden_size in [10,100,1000]:
tf.reset_default_graph()
model = Model5_mnist_bn(hidden_size=hidden_size)
taskname = 'mnist_m5_h%d_bnlr_T%d' % (hidden_size,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
lr_list=np.logspace(-3,6,40), max_step=6000,
#lr_list=np.logspace(-3,5,40), max_step=6000,
#lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')
for hidden_size in [10,100,1000]:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnlr_T%d.dat' % (hidden_size,i+0))
x = s[0]
value_history.append(s[1])
step = 6000
print(hidden_size)
#plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m5_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m5_bnlr')
%%writefile ./script/mnist_m5_bnsplr.py
from ModelAndTest import *
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
dataset = mnist
for i in range(5):
for lrab in [10,1,0.1]:
for hidden_size in [10,100,1000]:
tf.reset_default_graph()
model = Model5_mnist_bn_split(hidden_size=hidden_size)
model.learning_rate_ab = lrab
taskname = 'mnist_m5_h%d_bnsplr_ab%g_T%d' % (hidden_size,lrab,i)
#tensorboard_dir = '/home/*/Results/%s/' % taskname
tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname
test = Test()
test.test_lr(model=model, dataset=dataset,
#lr_list=np.logspace(-3,8,50), max_step=6000,
lr_list=np.logspace(-3,7,40), max_step=3000,
#lr_list=np.logspace(-3,7,40), max_step=6000,
#lr_list=np.logspace(-3,5,40), max_step=6000,
#lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
logdir=tensorboard_dir)
test.value_check()
data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')
for hidden_size in [10,100,1000]:
lr_list = [10,1,0.1]
values = []
for lr in lr_list:
value_history = []
for i in range(5):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnsplr_ab%g_T%d.dat' % (hidden_size,lr,i))
x = s[0]
value_history.append(s[1])
#values.append(np.mean(value_history, axis=0))
values.append(value_history)
step = 3000 # 1-epoch 600 step
step = 600 # 1-epoch 600 step
print(hidden_size)
# plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/mnist_m5_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/mnist_m5_bnsplr')
nTest = 5
values_pack = {}
for hidden_size in [10,100,1000]:
value_history = []
for i in range(nTest):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_gdlr_T%d.dat' % (hidden_size,i+0))
x = s[0]
value_history.append(s[1])
x_gd = x
vh_gd = value_history
value_history = []
for i in range(nTest):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnlr_T%d.dat' % (hidden_size,i+0))
x = s[0]
value_history.append(s[1])
x_bn = x
vh_bn = value_history
values = []
lr_list = [1,0.1,0.01,0.001]
lr_list = [100,10,1,0.1] #[1,0.1,0.01,0.001]
lr_list = [10,1,0.1] #[1,0.1,0.01,0.001]
for lr in lr_list:
value_history = []
for i in range(nTest+5):
s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnsplr_ab%g_T%d.dat' % (hidden_size,lr,i))
x = s[0]
value_history.append(s[1])
values.append(value_history)
x_bnsp = x
vhs_bnsp = values
step = 6000 # 1-epoch 600 step
values_pack[str(hidden_size)] = vh_gd, vh_bn, vhs_bnsp
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
fig = plt.figure(figsize=[16,8])
save_id = 60 # max 301, 60 is 1-epoch
col = 1
for hidden_size in [10,100,1000]:
vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
minloss = min(np.mean(vhs_bnsp[0],axis=0)[:,-1,col])
# minloss = 0
# plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'b-')
# plt.semilogx(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-.')
plt.loglog(x_bnsp, np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]-minloss,'--')
plt.plot(x_bnsp, np.mean(vhs_bnsp[1],axis=0)[:,save_id,col]-minloss,'-s')
plt.plot(x_bnsp, np.mean(vhs_bnsp[2],axis=0)[:,save_id,col]-minloss,'-o')
# plt.plot(x_bnsp, np.mean(vhs_bnsp[3],axis=0)[:,save_id,col],'--')
# plt.plot(x_bnsp, (np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]+
# np.mean(vhs_bnsp[1],axis=0)[:,save_id,col]+
# np.mean(vhs_bnsp[2],axis=0)[:,save_id,col]+
# np.mean(vhs_bnsp[3],axis=0)[:,save_id,col])/4,'b--')
# plt.ylim([0,0.1])
plt.xlim([1e-3,1e7]);
# plt.ylim([0.0,0.06])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend({'10','100','1000'})
x = np.linspace(0,2*np.pi,100)
y = np.sin(x) + np.random.random(100) * 0.8
def smooth(y, m):
ym = y+0.0
N = len(y)
ys = np.zeros([N+1])
for i in range(N):
ys[i+1] = ys[i] + y[i]
for i in range(N):
wid = min([m,i,N-1-i])
#ym[i] = sum(y[i-wid:i+wid+1]) / (2*wid+1)
ym[i] = (ys[i+wid+1] - ys[i-wid]) / (2*wid+1)
return ym
def smooth_vh(vh, m=10): # size = [5,40,601,5]
vh = np.array(vh)
n1,n2,n3,n4 = vh.shape
#m = 10;
for i1 in range(n1):
for i2 in range(n2):
for i4 in range(n4):
y = smooth(vh[i1,i2,:,i4],m)
#y = np.exp(smooth(np.log(vh[i1,i2,:,i4]),m))
vh[i1,i2,:,i4] = y
return vh
def smooth_log_vh(vh, m=10): # size = [5,40,601,5]
vh = np.array(vh)
n1,n2,n3,n4 = vh.shape
#m = 10;
for i1 in range(n1):
for i2 in range(n2):
for i4 in range(n4):
# y = smooth(vh[i1,i2,:,i4],m)
y = np.exp(smooth(np.log(vh[i1,i2,:,i4]),m))
vh[i1,i2,:,i4] = y
return vh
plt.plot(x, y,'o')
plt.plot(x, smooth(y,3), 'r-', lw=2)
plt.plot(x, smooth(y,5), 'g-', lw=2)
vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
kk=30
plt.semilogy(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],1),'r-')
# plt.plot(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],3),'g-')
plt.plot(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],20),'b-')
vhs_pack={}
for hidden_size in [10,100,1000]:
vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
vhs = smooth_vh(vhs_bnsp[0],ms)
vhs_pack[str(hidden_size)] = vhs
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1
fig = plt.figure(figsize=[6,4])
save_id = 60 # max 301, 60 is 1-epoch
col = 2
ms = 1 # smooth
for hidden_size in [10,100,1000]:
vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
# vhs = smooth_vh(vhs_bnsp[0],ms)
# vhs = smooth_log_vh(vhs_bnsp[0],ms)
vhs = vhs_pack[str(hidden_size)]
minloss = min(np.mean(vhs,axis=0)[:,300-ms,col])
minloss = max(np.mean(vhs,axis=0)[:,300-ms,col])
minloss = 0
# minloss = 1
# plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'b-')
# plt.semilogx(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-.')
v = np.mean(vhs,axis=0)[:,save_id,col]-minloss
print(v.T)
#v = np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]-minloss
# plt.loglog(x_bnsp, 1-v,'-o')
plt.semilogx(x_bnsp, v,'--o')
# plt.loglog(x_bnsp, v,'--o')
# plt.ylim([0,0.1])
# plt.xlim([1e-2,1e5]);
# plt.ylim([0.0,0.06])
plt.ylim([0.7,1.03])
plt.xlabel('learning rate')
# plt.ylabel('loss')
# plt.ylabel('accuracy')
plt.ylabel('accuracy (train)')
plt.grid()
# plt.legend(['hidden=10','hidden=100','hidden=1000'])
# plt.legend(['hidden=10','hidden=100','hidden=1000'],bbox_to_anchor=(.3, 0.9), loc=2, borderaxespad=0.)
print('save_id=',save_id,'col=',col,'ms=',ms)
plt.plot([2e0,2e3],np.array([0.,0])+0.95,'b',linewidth=4)
plt.plot([1.2e0,0.8e4],np.array([0.,0])+0.98,'r',linewidth=4)
plt.plot([7e-1,1.5e4],np.array([0.,0])+0.99,'g',linewidth=4)
# plt.plot([2e0,2e3],np.array([0.,0])+0.95,'b',linewidth=4)
# plt.plot([1e0,1e4],np.array([0.,0])+0.965,'r',linewidth=4)
# plt.plot([4e-1,4e4],np.array([0.,0])+1.0,'g',linewidth=4)
# plt.legend(['hidden=10','hidden=100','hidden=1000'],bbox_to_anchor=(.3, 0.4), loc=2, borderaxespad=0.)
plt.legend(['hidden=10','hidden=100','hidden=1000','wid=1e3','wid=1e4','wid=1e5'],bbox_to_anchor=(.3, 0.5), loc=2, borderaxespad=0.)
plt.tight_layout()
# plt.savefig("./figures/mnist_m5_compare_acc.pdf")